package com.offcn.bigdata.spark.p1;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;

/**
 * spark入门案例之java-lambda版
 */
public class JavaLambdaWordCountOps {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf();//相当于mr中的Configuration
        conf.setMaster("local[*]");
        conf.setAppName("JavaLambdaWordCount");
        JavaSparkContext jsc = new JavaSparkContext(conf);//相当于mr中的Job

        JavaRDD<String> linesRDD = jsc.textFile("F:/hello.txt");

        JavaRDD<String> wordsRDD = linesRDD.flatMap(line -> Arrays.asList(line.split("\\s+")).iterator());

        JavaPairRDD<String, Integer> pairsRDD = wordsRDD.mapToPair(word -> new Tuple2<String, Integer>(word, 1));

        JavaPairRDD<String, Integer> ret = pairsRDD.reduceByKey((v1, v2) -> v1 + v2);

        ret.foreach(kv -> System.out.printf("word: %s, count: %d\n", kv._1, kv._2));


        jsc.stop();//资源释放
    }
}
